#!/usr/bin/env python
# -*- coding: utf-8 -*-

# forever mako
'dataDownload module '

__author__ = 'jiahui'

import sys


import urllib,urllib2,codecs
import os,time ,threading,csv,shutil,datetime

import lockhelp as lh

flatlist = lambda L: sum(map(flatlist,L),[]) if isinstance(L,list) else [L]
#------------------------------------------------------------------------------------------------
class dataDownload:

    def __init__(self):
        self.lock = threading.Lock()
        self.codesz=[]
        self.codesh=[]
        self.MaxThreadNum=50
        self.downloadingTreadNum=0
        self.failcode=[]
        self.successcode=[]
        dt = datetime.datetime.now()-datetime.timedelta(days=1)
        self.datestr=dt.strftime('&f=%Y&d=%m&e%d')

    def loadcode(self,filename=''):
        if not filename:
            filename='%s/stockcode.csv' % (os.path.split(os.path.realpath(__file__))[0])
        if os.path.exists(filename):
            self.codesh=[]
            self.codesz=[]
            csvfile = codecs.open(filename, 'rU', 'utf-8')
            reader = csv.reader(csvfile)
            allcode = [ (n[:-3]) for n  in flatlist([c for c in reader]) if n[:-3].isdigit() and (n[0]=='6' or n[0]=='0')]
            for c in allcode:
                if c[0]=='6':
                    self.codesh.append(c)
                elif c[0]=='0':
                    self.codesz.append(c)
            csvfile.close() 

    def download(self):
        codelist=['%s.ss'% c for c in self.codesh]+['%s.sz'% c for c in self.codesz]
        self.downloadAll(codelist)

    def downloadAll(self,codelist,savedir=''):
        if not savedir:
            savedir ='%s/data' % (os.path.split(os.path.realpath(__file__))[0])
        self.failcode=[]
        self.successcode=[]
        if os.path.exists(savedir):
            shutil.rmtree(savedir)
        if not os.path.exists(savedir):
            os.makedirs(savedir)
        allThread=[]
        todel=[]
        # for testings
        #codelist = codelist[0:50]
        for sc in codelist:
            while True :
                threadNum=0
                with lh.lockhelp(self.lock):
                    threadNum = self.downloadingTreadNum
                if threadNum>=self.MaxThreadNum:
                    time.sleep(0.1)
                else:
                    break
            filename = '%s/%s.csv' %(savedir,sc)
            with lh.lockhelp(self.lock):
                self.downloadingTreadNum = self.downloadingTreadNum+1
            storeT=[]
            todel.append(storeT)
            t=threading.Thread(target=self.downloadOne,args=(sc,filename,self.defaultCompleteCallBack,
                                                             allThread,storeT))
            storeT.append(t)
            allThread.append(t)
            t.start()      
        while len(allThread)>0:
            f=len(self.failcode)
            s=len(self.successcode)
            p=int((f+s)*100/(len(self.codesz)+len(self.codesh)) )
            time.sleep(0.2)

    def defaultCompleteCallBack(self,code,file,result):
        with lh.lockhelp(self.lock):
            self.downloadingTreadNum = self.downloadingTreadNum-1
            if result:
                self.successcode.append(code)
            else:
                self.failcode.append(code)
            f=len(self.failcode)
            s=len(self.successcode)
            p=int((f+s)*100/(len(self.codesz)+len(self.codesh)) )
            print('DownLoading[%s]...Fail:%d,Success:%d............%d%%' % (str(code),f,s,p))  

    def downloadOne(self,sockcode,savefile='',completeCallBack=None,alltread=None,t=None):
        try:
            url = 'http://table.finance.yahoo.com/table.csv?s=%s%s' % (sockcode,self.datestr)
            if not savefile:
                savefile='%s/data/%s.csv' % (os.path.split(os.path.realpath(__file__))[0],sockcode)            
            f = urllib2.urlopen(url) 
            data = f.read() 
            with open(savefile, "wb") as code:  
                code.write(data)
            if completeCallBack:
                completeCallBack(sockcode,savefile,True)
        except BaseException as e:
            if completeCallBack:
                completeCallBack(sockcode,savefile,False)
        finally:
            if alltread and t:
                alltread.remove(t[0])
                

    def CheckCsvData(self,filename):
        if os.path.exists(filename):
            csvfile = codecs.open(filename, 'rU', 'utf-8')
            reader = csv.reader(csvfile)
            csvfile.close() 


#------------------------------------------------------------------------------------------------
if __name__=='__main__':
    ddd =dataDownload()
    ddd.loadcode()
    ddd.download()
    
    date_data=None
    filename ='%s/data/000001.sz.csv' % (os.path.split(os.path.realpath(__file__))[0]) 
    if os.path.exists(filename):
        csvfile = codecs.open(filename, 'rU', 'utf-8')
        reader = csv.reader(csvfile)    
        for Date,Open,High,Low,Close,Volume,AdjClose in reader:
            if reader.line_num==1:
                continue
            date_data=Date
            break
        csvfile.close()     
    
    if date_data:
        filename ='%s/data/date_data' % (os.path.split(os.path.realpath(__file__))[0])    
        date_datafile = file(filename, 'wb')
        date_datafile.write(date_data)
        date_datafile.close()    